package com.x.spider.util;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import java.io.IOException;
import java.util.Map;

public class SpiderUtils {

	private final static int requestTimeOut = 600000;// 请求超时时间

	private static Document getDocument(String url, Map<String, String> headers) {
		Connection conn = Jsoup.connect(url);
		if (headers != null) {
			conn.headers(headers);
		}
		try {
			return conn.timeout(requestTimeOut).get();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return null;
	}

	public static void main(String[] args) throws IOException {
		WebClient webClient = new WebClient(BrowserVersion.CHROME);
		webClient.getOptions().setCssEnabled(false);
		HtmlPage page = webClient.getPage("https://www.che168.com/beijing/list/");
		System.out.println(page.asText());
	}
}
